package com.crawler.repitle;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class IndustryCrawler {

    private ExecutorService executorService = Executors.newFixedThreadPool(10);
    private ParquetSender sender;

    public IndustryCrawler() {
    }
    /**
     * 正则表达式，取=和|之间的字符串，不包括=和|
     */
    private Pattern pattern = Pattern.compile("共(.*?)页，到第");
    WebDriver driver = null;

    public void init(){
        String driverPath = App.class.getResource("/" ) + "exe/chromedriver.exe";
        driverPath = driverPath.substring(6);
        System.setProperty("webdriver.chrome.driver", driverPath);
        ChromeOptions options = new ChromeOptions();
        //浏览器后台运行
        options.addArguments("headless");
        driver = new ChromeDriver(options);
        sender = new ParquetSender();
        sender.init();
    }
    public void crawl() throws Exception {
        String url = "https://jobs.51job.com/";
        driver.get(url);
        Document doc = Jsoup.parse(driver.getPageSource());
        Element mainCenter = doc.getElementsByClass("maincenter").first();
        Element filter = mainCenter.getElementsByClass("filter").get(1);
        Elements classes = filter.getElementsByClass("e5");

        for (int i = 0; i < classes.size(); i++) {
            Element p = classes.get(i).getElementsByTag("p").first();
            Element industryClass = p.getElementsByTag("a").first();
            String industryClassHref = industryClass.attr("href");
            String industryClassName = industryClass.getElementsByTag("strong").first().ownText();
            Element lkst = classes.get(i).getElementsByClass("lkst").first();
            Elements industrys = lkst.getElementsByTag("a");
            for (int j = 0; j < industrys.size(); j++) {
                Element industry = industrys.get(j);
                final String industryHref = industry.attr("href");
                final String industryName = industry.ownText();
                doPageCrawl(industryName, industryHref, industryHref);

            }
        }

    }

    private List<Map<String, String>> doPageCrawl(String industryName, String industryHref, String pageUrl){
        driver.get(pageUrl);
        Document doc = Jsoup.parse(driver.getPageSource());
        Elements elements = doc.getElementsByClass("dw_page");
        Elements tds = elements.get(0).getElementsByClass("td");
        Element pageNumberElement = tds.get(0);
        Matcher m = pattern.matcher(pageNumberElement.ownText());
        List<Map<String, String>> list = new ArrayList<Map<String, String>>();
        if(m.find()) {
            int count = Integer.valueOf(m.group(1));
            for (int i = 1; i <= count; i++) {
                String subUrl = "https://jobs.51job.com/gaojiruanjian/p" + i + "/";
                driver.get(subUrl);
                Document realDoc = Jsoup.parse(driver.getPageSource());
                Element mainCenter = realDoc.getElementsByClass("maincenter").first();
                Element left = mainCenter.getElementsByClass("left").first();
                Element gbox = left.getElementsByClass("gbox").first();
                Elements elElements = gbox.getElementsByClass("e");
                for (int j = 1; j < elElements.size(); j ++) {
                    Map<String, String> map = new HashMap<String, String>();
                    Element element = elElements.get(j);
                    Element info = element.getElementsByClass("info").first();
                    //职位名称
                    Element t1 = info.getElementsByClass("title").first();
                    Element job = t1.getElementsByTag("a").first();
                    map.put("jobName", job.ownText());
                    map.put("jobHref", job.attr("href"));
                    //公司名称
                    Element company = element.getElementsByClass("name").first();
                    map.put("companyName", company.ownText());
                    map.put("companyHref", company.attr("href"));
                    //工作地点
                    Element location = element.getElementsByClass("location name").first();
                    map.put("jobLocation", location.ownText());
                    //工资
                    Element salary = element.getElementsByClass("location").get(1);
                    map.put("salary",salary.ownText());
                    //发布时间
                    Element time = element.getElementsByClass("time").first();
                    map.put("time", time.ownText());

                    //年限，之类的组合
                    Element order = element.getElementsByClass("order").first();
                    map.put("order", order.toString());

                    Element text = element.getElementsByClass("text").first();
                    map.put("intro", text.ownText());

                    map.put("industryName", industryName);
                    map.put("industryHref", industryHref);


                    try {
                        sender.send(map);
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }

        }
        return list;

    }

    public void destroy() {
        driver.close();
        driver.quit();
        try {
            sender.destroy();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
